home *** CD-ROM | disk | FTP | other *** search
/ Personal Computer World 2008 February / PCWFEB08.iso / Software / Freeware / Miro 1.0 / Miro_Installer.exe / Miro_Downloader.exe / test / httpclienttest.pyc (.txt) < prev    next >
Encoding:
Python Compiled Bytecode  |  2007-11-12  |  55.0 KB  |  1,339 lines
  1. # Source Generated with Decompyle++
  2. # File: in.pyc (Python 2.5)
  3.  
  4. import unittest
  5. import email.Utils as email
  6. import socket
  7. import tempfile
  8. import traceback
  9. from copy import copy
  10. from StringIO import StringIO
  11. from clock import clock
  12. import os
  13. from download_utils import cleanFilename
  14. import download_utils
  15. import database
  16. import dialogs
  17. import httpclient
  18. import util
  19. from framework import EventLoopTest, DemocracyTestCase, HadToStopEventLoop
  20.  
  21. class TestingConnectionHandler(httpclient.ConnectionHandler):
  22.     
  23.     def __init__(self, test):
  24.         super(TestingConnectionHandler, self).__init__()
  25.         self.states['foo'] = self.handleFoo
  26.         self.states['bar'] = self.handleBar
  27.         self.states['noread'] = None
  28.         self.fooData = ''
  29.         self.barData = ''
  30.         self.gotHandleClose = False
  31.         self.closeType = None
  32.         self.test = test
  33.  
  34.     
  35.     def handleFoo(self):
  36.         data = self.buffer.read()
  37.         self.fooData += data
  38.         self.test.stopEventLoop(False)
  39.  
  40.     
  41.     def handleBar(self):
  42.         data = self.buffer.read()
  43.         self.barData += data
  44.         self.test.stopEventLoop(False)
  45.  
  46.     
  47.     def handleClose(self, type):
  48.         self.gotHandleClose = True
  49.         self.closeType = type
  50.         self.test.stopEventLoop(False)
  51.  
  52.  
  53.  
  54. class FakeStream:
  55.     
  56.     def __init__(self, closeCallback = None):
  57.         self.open = False
  58.         self.readCallback = None
  59.         self.closeCallback = closeCallback
  60.         self.timedOut = False
  61.         self.connectionErrback = None
  62.         self.name = ''
  63.         self.output = ''
  64.         self.unprocessed = ''
  65.         self.input = ''
  66.         self.pendingOutput = ''
  67.         self.timedOut = False
  68.         self.pages = {
  69.             'participatoryculture.org': {
  70.                 '/democracytest/normalpage.txt': 'I AM A NORMAL PAGE\n',
  71.                 '/democracytest/normalpage2.txt': 'I AM A NORMAL PAGE\n',
  72.                 '/democracytest/normalpage3.txt': 'I AM A NORMAL PAGE\n',
  73.                 '/democracytest/nohead.php': 'DYNAMIC CONTENT' },
  74.             'jigsaw.w3.org': {
  75.                 '/HTTP/Basic/': 'normal page',
  76.                 '/HTTP/Digest/': 'normal page' },
  77.             'www.bar.com': {
  78.                 '/': 'Normal',
  79.                 '/2': 'Blah' },
  80.             'www.baz.com': {
  81.                 '/': 'Normal',
  82.                 '/2': 'Blah' },
  83.             'www.froz.com': {
  84.                 '/': 'Normal',
  85.                 '/2': 'Blah' },
  86.             'www.qux.com': {
  87.                 '/': 'Normal',
  88.                 '/2': 'Blah' } }
  89.         self.noheadPages = {
  90.             'participatoryculture.org': [
  91.                 '/democracytest/nohead.php'] }
  92.         self.basicAuthPages = {
  93.             'jigsaw.w3.org': {
  94.                 '/HTTP/Basic/': 'Basic Z3Vlc3Q6Z3Vlc3Q=' } }
  95.         self.digestAuthPages = {
  96.             'jigsaw.w3.org': {
  97.                 '/HTTP/Digest/': 'STUFF GOES HERE' } }
  98.  
  99.     
  100.     def _tryReadCallback(self):
  101.         if len(self.pendingOutput) > 0 and self.readCallback:
  102.             response = self.pendingOutput
  103.             self.pendingOutput = ''
  104.             self.readCallback(response)
  105.         
  106.  
  107.     
  108.     def _generateResponse(self, method, uri, version, headers):
  109.         text = None
  110.         now = email.Utils.formatdate(usegmt = True)
  111.         if self.pages.has_key(headers['Host']):
  112.             host_pages = self.pages[headers['Host']]
  113.             if host_pages.has_key(uri):
  114.                 text = host_pages[uri]
  115.             
  116.         else:
  117.             self.errback(httpclient.ConnectionError("Can't connect"))
  118.             return None
  119.         if text is not None:
  120.             if method == 'GET':
  121.                 if self.basicAuthPages.has_key(headers['Host']) and uri in self.basicAuthPages[headers['Host']].keys():
  122.                     pass
  123.                 None if not headers.has_key('Authorization') or self.basicAuthPages[headers['Host']][uri] != headers['Authorization'] else self.basicAuthPages[headers['Host']][uri] != headers['Authorization']
  124.                 if self.digestAuthPages.has_key(headers['Host']) and uri in self.digestAuthPages[headers['Host']].keys():
  125.                     pass
  126.                 None if not headers.has_key('Authorization') or self.digestAuthPages[headers['Host']][uri] != headers['Authorization'] else self.digestAuthPages[headers['Host']][uri] != headers['Authorization']
  127.                 return 'HTTP/1.1 200 OK\r\nContent-Type: text/plain; charset=UTF-8\r\nLast-Modified: %s\r\nDate: %s\r\nContent-Length: %d\r\n\r\n%s' % (now, now, len(text), text)
  128.             elif method == 'HEAD':
  129.                 if self.noheadPages.has_key(headers['Host']) and uri in self.noheadPages[headers['Host']]:
  130.                     return 'HTTP/1.1 405 NOT ALLOWED\r\nDate: %s\r\n\r\n' % now
  131.                 else:
  132.                     return 'HTTP/1.1 200 OK\r\nContent-Type: text/plain; charset=UTF-8\r\nLast-Modified: %s\r\nDate: %s\r\nContent-Length: %d\r\n\r\n' % (now, now, len(text))
  133.             
  134.         
  135.         text = '<h1>Not found</h1>'
  136.         return 'HTTP/1.1 404 Not Found\r\nContent-Type: text/html; charset=UTF-8\r\nDate: %s\r\nContent-Length: %d\r\n\r\n%s' % (now, len(text), text)
  137.  
  138.     
  139.     def _processRequest(self, method, uri, version, headers):
  140.         response = self._generateResponse(method, uri, version, headers)
  141.         if response is not None:
  142.             self.pendingOutput += response
  143.             self._tryReadCallback()
  144.         
  145.  
  146.     
  147.     def _processData(self, data):
  148.         self.unprocessed += data
  149.         for x in headers:
  150.             headers = _[1](_[1][x.split(': ', 1)])
  151.             self._processRequest(request_method, request_uri, request_version, headers)
  152.             []
  153.  
  154.     
  155.     def __str__(self):
  156.         if self.name:
  157.             return '%s: %s' % (type(self).__name__, self.name)
  158.         else:
  159.             return 'Unknown %s' % (type(self).__name__,)
  160.  
  161.     
  162.     def startReadTimeout(self):
  163.         pass
  164.  
  165.     
  166.     def stopReadTimeout(self):
  167.         pass
  168.  
  169.     
  170.     def openConnection(self, host, port, callback, errback, disabledReadTimeout = None):
  171.         self.name = 'Outgoing %s:%s' % (host, port)
  172.         self.output = ''
  173.         self.host = host
  174.         self.port = port
  175.         self.open = True
  176.         self.errback = errback
  177.         self.dsiabledReadTimeout = disabledReadTimeout
  178.         callback(self)
  179.  
  180.     
  181.     def acceptConnection(self, host, port, callback, errback):
  182.         errback()
  183.  
  184.     
  185.     def closeConnection(self):
  186.         self.open = False
  187.  
  188.     
  189.     def isOpen(self):
  190.         return self.open
  191.  
  192.     
  193.     def sendData(self, data, callback = None):
  194.         if not self.isOpen():
  195.             raise ValueError('Socket not connected')
  196.         
  197.         self.output += data
  198.         self._processData(data)
  199.  
  200.     
  201.     def startReading(self, readCallback):
  202.         if not self.isOpen():
  203.             raise ValueError('Socket not connected')
  204.         
  205.         self.readCallback = readCallback
  206.         self._tryReadCallback()
  207.  
  208.     
  209.     def stopReading(self):
  210.         '''Stop reading from the socket.'''
  211.         if not self.isOpen():
  212.             raise ValueError('Socket not connected')
  213.         
  214.         self.readCallback = None
  215.  
  216.     
  217.     def onReadTimeout(self):
  218.         raise IOError('Read Timeout')
  219.  
  220.     
  221.     def handleSocketError(self, code, msg, operation):
  222.         raise IOError('Socket Error')
  223.  
  224.  
  225.  
  226. class DumbFakeStream(FakeStream):
  227.     
  228.     def _generateResponse(self, method, uri, version, headers):
  229.         pass
  230.  
  231.  
  232.  
  233. class TestingHTTPConnection(httpclient.HTTPConnection):
  234.     """HTTPConnection that doesn't actually connect to the network."""
  235.     streamFactory = FakeStream
  236.  
  237.  
  238. class TestingHTTPSConnection(httpclient.HTTPSConnection):
  239.     """HTTPSConnection that doesn't actually connect to the network."""
  240.     streamFactory = FakeStream
  241.  
  242.  
  243. class DumbTestingHTTPConnection(httpclient.HTTPConnection):
  244.     """HTTPConnection that doesn't actually do much of anything."""
  245.     streamFactory = DumbFakeStream
  246.  
  247.  
  248. class TestingHTTPConnectionPool(httpclient.HTTPConnectionPool):
  249.     MAX_CONNECTIONS = 4
  250.     HTTP_CONN = TestingHTTPConnection
  251.     HTTPS_CONN = TestingHTTPSConnection
  252.     
  253.     def getConnection(self, scheme, host, port = None, type = 'active'):
  254.         if port is None:
  255.             if scheme == 'https':
  256.                 port = 443
  257.             else:
  258.                 port = 80
  259.         
  260.         conns = self._getServerConnections(scheme, host, port)
  261.         for conn in conns[type]:
  262.             return conn
  263.         
  264.  
  265.     
  266.     def assertConnectionStarted(self, url):
  267.         if not self.checkConnectionStarted(url):
  268.             raise AssertionError
  269.  
  270.     
  271.     def assertConnectionNotStarted(self, url):
  272.         if not not self.checkConnectionStarted(url):
  273.             raise AssertionError
  274.  
  275.     
  276.     def checkConnectionStarted(self, url):
  277.         (scheme, host, port, path) = httpclient.parseURL(url)
  278.         conns = self._getServerConnections(scheme, host, port)
  279.         for conn in conns['active']:
  280.             
  281.             try:
  282.                 if conn.host == host and conn.port == port:
  283.                     return True
  284.             continue
  285.             continue
  286.  
  287.         
  288.         return False
  289.  
  290.     
  291.     def closeConnection(self, scheme, host, port = None, type = 'active'):
  292.         conn = self.getConnection(scheme, host, port, type)
  293.         conn.handleClose(socket.SHUT_RDWR)
  294.  
  295.  
  296.  
  297. class DumbTestingHTTPConnectionPool(TestingHTTPConnectionPool):
  298.     HTTP_CONN = DumbTestingHTTPConnection
  299.     HTTPS_CONN = DumbTestingHTTPConnection
  300.  
  301.  
  302. class DumbTestHTTPClient(httpclient.HTTPClient):
  303.     connectionPool = DumbTestingHTTPConnectionPool()
  304.  
  305.  
  306. class TestHTTPClient(httpclient.HTTPClient):
  307.     connectionPool = TestingHTTPConnectionPool()
  308.  
  309.  
  310. class TestingHeaderGrabber(httpclient.HTTPHeaderGrabber):
  311.     connectionPool = TestingHTTPConnectionPool()
  312.  
  313.  
  314. class TestingAuthDelegate:
  315.     
  316.     def __init__(self):
  317.         self.logins = []
  318.  
  319.     
  320.     def addLogin(self, user, password):
  321.         self.logins.append((user, password))
  322.  
  323.     
  324.     def runDialog(self, dialog):
  325.         if self.logins:
  326.             (user, password) = self.logins.pop(0)
  327.             dialog.runCallback(dialogs.BUTTON_OK, user, password)
  328.         else:
  329.             dialog.runCallback(None)
  330.  
  331.  
  332.  
  333. def startResponse(version = '1.1', status = 200, headers = { }):
  334.     rv = 'HTTP/%s %s OK\r\nContent-Type: text/plain; charset=ISO-8859-1\r\nLast-Modified: Wed, 10 May 2006 22:30:33 GMT\r\nDate: Wed, 10 May 2006 22:38:39 GMT\r\n' % (version, status)
  335.     for key, value in headers.items():
  336.         rv += '%s: %s\r\n' % (key, value)
  337.     
  338.     rv += '\r\n'
  339.     return rv
  340.  
  341.  
  342. class AsyncSocketTest(EventLoopTest):
  343.     
  344.     def setUp(self):
  345.         self.data = None
  346.         self.errbackCalled = False
  347.         self.callbackCalled = False
  348.         self.fakeCallbackError = False
  349.         EventLoopTest.setUp(self)
  350.  
  351.     
  352.     def callback(self, data):
  353.         if self.fakeCallbackError:
  354.             1 / 0
  355.         
  356.         self.data = data
  357.         self.callbackCalled = True
  358.         self.stopEventLoop(False)
  359.  
  360.     
  361.     def errback(self, error):
  362.         self.data = error
  363.         self.errbackCalled = True
  364.         self.stopEventLoop(False)
  365.  
  366.  
  367.  
  368. class NetworkBufferTest(DemocracyTestCase):
  369.     
  370.     def setUp(self):
  371.         self.buffer = httpclient.NetworkBuffer()
  372.  
  373.     
  374.     def testReadLine(self):
  375.         self.buffer.addData('HEL')
  376.         self.assertEquals(self.buffer.readline(), None)
  377.         self.buffer.addData('LO\r\n')
  378.         self.assertEquals(self.buffer.readline(), 'HELLO')
  379.         self.buffer.addData('HOWS\r\nIT\nGOING\r\nCRONLY\rDOESNTBREAK')
  380.         self.assertEquals(self.buffer.readline(), 'HOWS')
  381.         self.assertEquals(self.buffer.readline(), 'IT')
  382.         self.assertEquals(self.buffer.readline(), 'GOING')
  383.         self.assertEquals(self.buffer.readline(), None)
  384.         self.assertEquals(self.buffer.read(), 'CRONLY\rDOESNTBREAK')
  385.  
  386.     
  387.     def testRead(self):
  388.         self.buffer.addData('12345678901234567890')
  389.         self.assertEquals(self.buffer.read(4), '1234')
  390.         self.assertEquals(self.buffer.read(6), '567890')
  391.         self.buffer.addData('CARBOAT')
  392.         self.assertEquals(self.buffer.read(), '1234567890CARBOAT')
  393.  
  394.     
  395.     def testLength(self):
  396.         self.buffer.addData('ONE\r\nTWO')
  397.         self.assertEquals(self.buffer.length, 8)
  398.         self.buffer.readline()
  399.         self.assertEquals(self.buffer.length, 3)
  400.         self.buffer.read(1)
  401.         self.assertEquals(self.buffer.length, 2)
  402.         self.buffer.unread('AAA')
  403.         self.assertEquals(self.buffer.length, 5)
  404.         self.buffer.addData('MORE')
  405.         self.assertEquals(self.buffer.length, 9)
  406.  
  407.     
  408.     def testGetValue(self):
  409.         self.buffer.addData('ONE')
  410.         self.buffer.addData('TWO')
  411.         self.buffer.addData('THREE')
  412.         self.assertEquals(self.buffer.getValue(), 'ONETWOTHREE')
  413.         self.assertEquals(self.buffer.getValue(), 'ONETWOTHREE')
  414.  
  415.  
  416.  
  417. class WeirdCloseConnectionTest(AsyncSocketTest):
  418.     
  419.     def testCloseDuringOpenConnection(self):
  420.         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  421.         sock.bind(('127.0.0.1', 0))
  422.         sock.listen(1)
  423.         (host, port) = sock.getsockname()
  424.         
  425.         try:
  426.             conn = httpclient.AsyncSocket()
  427.             conn.openConnection(host, port, self.callback, self.errback)
  428.             conn.closeConnection()
  429.             self.runEventLoop(timeout = 1, timeoutNormal = True)
  430.             self.assert_(not (self.callbackCalled))
  431.             self.assert_(self.errbackCalled)
  432.         finally:
  433.             sock.close()
  434.  
  435.  
  436.     
  437.     def testCloseDurringAcceptConnection(self):
  438.         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  439.         
  440.         try:
  441.             conn = httpclient.AsyncSocket()
  442.             conn.acceptConnection('127.0.0.1', 0, self.callback, self.errback)
  443.             sock.connect((conn.addr, conn.port))
  444.             conn.closeConnection()
  445.             self.runEventLoop(timeout = 1, timeoutNormal = True)
  446.             self.assert_(not (self.callbackCalled))
  447.             self.assert_(self.errbackCalled)
  448.         finally:
  449.             sock.close()
  450.  
  451.  
  452.  
  453.  
  454. class ConnectionHandlerTest(EventLoopTest):
  455.     
  456.     def setUp(self):
  457.         EventLoopTest.setUp(self)
  458.         server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  459.         server.bind(('127.0.0.1', 0))
  460.         server.listen(1)
  461.         address = server.getsockname()
  462.         self.connectionHandler = TestingConnectionHandler(self)
  463.         
  464.         def stopEventLoop(conn):
  465.             self.stopEventLoop(False)
  466.  
  467.         self.connectionHandler.openConnection(address[0], address[1], stopEventLoop, stopEventLoop)
  468.         self.runEventLoop()
  469.         (self.remoteSocket, address) = server.accept()
  470.         self.remoteSocket.setblocking(False)
  471.  
  472.  
  473.  
  474. class DumbHTTPClientTest(AsyncSocketTest):
  475.     
  476.     def setUp(self):
  477.         AsyncSocketTest.setUp(self)
  478.         self.testRequest = DumbTestingHTTPConnection()
  479.         self.testRequest.openConnection('foo.com', 80, (lambda x: None), (lambda x: None))
  480.         self.testRequest.sendRequest(self.callback, self.errback, '', 80, method = 'GET', path = '/bar/baz;123?a=b')
  481.         self.authDelegate = TestingAuthDelegate()
  482.         dialogs.setDelegate(self.authDelegate)
  483.  
  484.     
  485.     def tearDown(self):
  486.         AsyncSocketTest.tearDown(self)
  487.  
  488.     
  489.     def testScheme(self):
  490.         conn = httpclient.HTTPConnection()
  491.         self.assertEquals(conn.scheme, 'http')
  492.  
  493.     
  494.     def testRequestLine(self):
  495.         self.assertEquals(self.testRequest.stream.output.split('\r\n')[0], 'GET /bar/baz;123?a=b HTTP/1.1')
  496.  
  497.     
  498.     def testStatusLine(self):
  499.         self.testRequest.handleData('HTTP/1.0 200 OK\r\n')
  500.         self.assertEquals(self.testRequest.version, 'HTTP/1.0')
  501.         self.assertEquals(self.testRequest.status, 200)
  502.         self.assertEquals(self.testRequest.reason, 'OK')
  503.  
  504.     
  505.     def testStatusLine2(self):
  506.         self.testRequest.handleData('HTTP/1.1 404 Not Found\r\n')
  507.         self.assertEquals(self.testRequest.version, 'HTTP/1.1')
  508.         self.assertEquals(self.testRequest.status, 404)
  509.         self.assertEquals(self.testRequest.reason, 'Not Found')
  510.  
  511.     
  512.     def testBadStatusLine(self):
  513.         self.testRequest.handleData('HTTP/0.9 200 OK\r\n')
  514.         self.assert_(self.errbackCalled)
  515.         self.assert_(isinstance(self.data, httpclient.BadStatusLine))
  516.  
  517.     
  518.     def testBadStatusLine2(self):
  519.         self.testRequest.handleData('HTTP/1.0 641 OK\r\n')
  520.         self.assert_(self.errbackCalled)
  521.         self.assert_(isinstance(self.data, httpclient.BadStatusLine))
  522.  
  523.     
  524.     def testBadStatusLine3(self):
  525.         self.testRequest.handleData('HTTP/1.0 TwoHundred OK\r\n')
  526.         self.assert_(self.errbackCalled)
  527.         self.assert_(isinstance(self.data, httpclient.BadStatusLine))
  528.  
  529.     
  530.     def testNoReason(self):
  531.         self.testRequest.handleData('HTTP/1.0 200\r\n')
  532.         self.assertEquals(self.testRequest.version, 'HTTP/1.0')
  533.         self.assertEquals(self.testRequest.status, 200)
  534.         self.assertEquals(self.testRequest.reason, '')
  535.  
  536.     
  537.     def testTryToHandleHTTP0Point9(self):
  538.         self.testRequest.handleData('StartOfThebody\r\n')
  539.         self.assertEquals(self.testRequest.version, 'HTTP/0.9')
  540.         self.assertEquals(self.testRequest.status, 200)
  541.         self.assertEquals(self.testRequest.reason, '')
  542.         self.assertEquals(self.testRequest.buffer.read(), 'StartOfThebody\r\n')
  543.  
  544.     fakeResponse = 'HTTP/1.0 200 OK\r\nContent-Type: text/plain; charset=ISO-8859-1\r\nLast-Modified: Wed, 10 May 2006 22:30:33 GMT\r\nDate: Wed, 10 May 2006 22:38:39 GMT\r\nX-Cache: HIT from pcf2.pcf.osuosl.org\r\nServer: Apache\r\nContent-Length: 14\r\n\r\nHELLO: WORLD\r\n'
  545.     
  546.     def testBasicHeaders(self):
  547.         self.testRequest.handleData(self.fakeResponse)
  548.         self.testRequest.handleClose(socket.SHUT_RD)
  549.         headers = self.testRequest.headers
  550.         self.assertEquals(headers['x-cache'], 'HIT from pcf2.pcf.osuosl.org')
  551.         self.assertEquals(headers['server'], 'Apache')
  552.         self.assertEquals(headers['last-modified'], 'Wed, 10 May 2006 22:30:33 GMT')
  553.         self.assertEquals(headers['content-length'], '14')
  554.         self.assertEquals(self.testRequest.contentLength, 14)
  555.         self.assertEquals(headers['date'], 'Wed, 10 May 2006 22:38:39 GMT')
  556.         self.assertEquals(headers['content-type'], 'text/plain; charset=ISO-8859-1')
  557.  
  558.     
  559.     def testCallbackError(self):
  560.         self.fakeCallbackError = True
  561.         self.failedCalled = False
  562.         
  563.         def fakeFailed(*args, **kwargs):
  564.             self.failedCalled = True
  565.  
  566.         oldFailed = util.failed
  567.         util.failed = fakeFailed
  568.         
  569.         try:
  570.             self.testRequest.handleData(self.fakeResponse)
  571.             self.testRequest.handleClose(socket.SHUT_RD)
  572.         finally:
  573.             util.failed = oldFailed
  574.  
  575.         self.assert_(self.failedCalled)
  576.  
  577.     
  578.     def testHeaderContinuation(self):
  579.         self.testRequest.handleData('HTTP/1.0 200 OK\r\n')
  580.         self.testRequest.handleData('Cont\r\n')
  581.         self.testRequest.handleData(' ent-Type: text/plain\r\n')
  582.         self.assertEquals(self.testRequest.headers['content-type'], 'text/plain')
  583.  
  584.     
  585.     def testHeaderJoin(self):
  586.         self.testRequest.handleData('HTTP/1.0 200 OK\r\n')
  587.         self.testRequest.handleData('x-test-list: 1\r\n')
  588.         self.testRequest.handleData('x-test-list: 2\r\n')
  589.         self.testRequest.handleData('x-test-list: 3\r\n')
  590.         self.testRequest.handleData('x-test-list: 4\r\n')
  591.         self.testRequest.handleData('\r\n')
  592.         self.assertEquals(self.testRequest.headers['x-test-list'], '1,2,3,4')
  593.  
  594.     
  595.     def testBadHeaderContinuation(self):
  596.         self.testRequest.handleData('HTTP/1.0 200 OK\r\n')
  597.         self.testRequest.handleData('IShouldBeContinued\r\n')
  598.         self.testRequest.handleData('\r\n')
  599.         self.assert_(self.errbackCalled)
  600.         self.assert_(isinstance(self.data, httpclient.BadHeaderLine))
  601.  
  602.     
  603.     def testWillClose(self):
  604.         self.testRequest.handleData(startResponse(headers = {
  605.             'Content-Length': 128 }))
  606.         self.assertEquals(self.testRequest.willClose, False)
  607.  
  608.     
  609.     def testWillClose2(self):
  610.         self.testRequest.handleData(startResponse(headers = {
  611.             'Transfer-Encoding': 'chunked' }))
  612.         self.assertEquals(self.testRequest.willClose, False)
  613.  
  614.     
  615.     def testWillClose3(self):
  616.         self.testRequest.handleData(startResponse(version = '1.0', headers = {
  617.             'Content-Length': 128 }))
  618.         self.assertEquals(self.testRequest.willClose, True)
  619.  
  620.     
  621.     def testWillClose4(self):
  622.         self.testRequest.handleData(startResponse())
  623.         self.assertEquals(self.testRequest.willClose, True)
  624.  
  625.     
  626.     def testWillClose5(self):
  627.         self.testRequest.handleData(startResponse(headers = {
  628.             'Connection': 'close',
  629.             'Content-Length': 128 }))
  630.         self.assertEquals(self.testRequest.willClose, True)
  631.  
  632.     
  633.     def testWillClose6(self):
  634.         self.testRequest.handleData(startResponse(headers = {
  635.             'Connection': 'CLoSe',
  636.             'Content-Length': 128 }))
  637.         self.assertEquals(self.testRequest.willClose, True)
  638.  
  639.     
  640.     def testPipeline(self):
  641.         self.assertEqual(self.testRequest.pipelinedRequest, None)
  642.         self.testRequest.handleData(startResponse(headers = {
  643.             'Content-Length': 128 }))
  644.         self.assertEquals(self.testRequest.canSendRequest(), True)
  645.         self.testRequest.sendRequest(self.callback, self.errback, '', 80, path = '/pipelined/path')
  646.         self.assertEquals(self.testRequest.pipelinedRequest[6], '/pipelined/path')
  647.         self.testRequest.handleData('a' * 128)
  648.         self.assert_(self.callbackCalled)
  649.         self.assertEquals(self.data['body'], 'a' * 128)
  650.         self.assertEquals(self.testRequest.state, 'response-status')
  651.         self.assertEquals(self.testRequest.path, '/pipelined/path')
  652.         self.assertEquals(self.testRequest.pipelinedRequest, None)
  653.         self.assertEquals(self.testRequest.headers, { })
  654.         self.assertEquals(self.testRequest.body, '')
  655.         self.assertEquals(self.testRequest.status, None)
  656.  
  657.     
  658.     def testBadPipeline(self):
  659.         self.testRequest.handleData(startResponse())
  660.         self.assertEquals(self.testRequest.canSendRequest(), False)
  661.         self.assertRaises(httpclient.NetworkError, self.testRequest.sendRequest, self.callback, self.errback, '', 80)
  662.  
  663.     
  664.     def testPipelineNeverStarted(self):
  665.         self.pipelineError = None
  666.         self.testRequest.handleData(startResponse(headers = {
  667.             'Content-Length': 128 }))
  668.         
  669.         def pipelineErrback(error):
  670.             self.pipelineError = error
  671.  
  672.         self.testRequest.sendRequest(self.callback, pipelineErrback, '', 80, path = '/pipelined/path')
  673.         self.testRequest.handleClose(socket.SHUT_RDWR)
  674.         self.assert_(isinstance(self.pipelineError, httpclient.PipelinedRequestNeverStarted))
  675.  
  676.     
  677.     def testPipelineNeverStarted2(self):
  678.         self.pipelineError = None
  679.         self.testRequest.handleData(startResponse(headers = {
  680.             'Content-Length': 128 }))
  681.         
  682.         def pipelineErrback(error):
  683.             self.pipelineError = error
  684.  
  685.         self.testRequest.sendRequest(self.callback, pipelineErrback, '', 80, path = '/pipelined/path')
  686.         self.testRequest.closeConnection()
  687.         self.assert_(isinstance(self.pipelineError, httpclient.PipelinedRequestNeverStarted))
  688.  
  689.     
  690.     def testContentLengthHandling(self):
  691.         self.testRequest.handleData(startResponse(headers = {
  692.             'Content-Length': '5' }))
  693.         self.testRequest.handleData('12345EXTRASTUFF')
  694.         self.assertEquals(self.testRequest.body, '12345')
  695.  
  696.     
  697.     def testTransferEncodingTrumpsContentLength(self):
  698.         self.testRequest.handleData(startResponse(headers = {
  699.             'Content-Length': '5',
  700.             'Transfer-Encoding': 'chunked' }))
  701.         self.assertEquals(self.testRequest.contentLength, None)
  702.  
  703.     
  704.     def test416ContentLength(self):
  705.         '''Test the content length after a 416 status code.'''
  706.         self.testRequest.handleData(startResponse(status = 416, headers = {
  707.             'Content-Range': 'bytes */1234' }))
  708.         self.assertEquals(self.testRequest.contentLength, 1234)
  709.  
  710.     
  711.     def testNoBody(self):
  712.         self.testRequest.handleData(startResponse(status = 204))
  713.         self.assertEquals(self.testRequest.state, 'closed')
  714.         self.assertEquals(self.testRequest.body, '')
  715.  
  716.     
  717.     def testNoBody2(self):
  718.         self.testRequest.handleData(startResponse(status = 123))
  719.         self.assertEquals(self.testRequest.state, 'closed')
  720.         self.assertEquals(self.testRequest.body, '')
  721.  
  722.     
  723.     def testNoBody3(self):
  724.         self.testRequest.method = 'HEAD'
  725.         self.testRequest.handleData(startResponse())
  726.         self.assertEquals(self.testRequest.state, 'closed')
  727.         self.assertEquals(self.testRequest.body, '')
  728.  
  729.     
  730.     def testNoBody4(self):
  731.         
  732.         self.testRequest.bodyDataCallback = lambda data: 0
  733.         self.testRequest.handleData(startResponse(headers = {
  734.             'Content-Length': '0' }))
  735.         self.assertEquals(self.testRequest.state, 'ready')
  736.         self.assertEquals(self.testRequest.body, '')
  737.  
  738.     
  739.     def testSplitUpMessage(self):
  740.         data = self.fakeResponse
  741.         for cutoff in [
  742.             3,
  743.             6,
  744.             10,
  745.             4,
  746.             100,
  747.             52]:
  748.             self.testRequest.handleData(data[:cutoff])
  749.             data = data[cutoff:]
  750.         
  751.         self.testRequest.handleData(data)
  752.         self.testRequest.handleClose(socket.SHUT_RD)
  753.         self.assertEquals(self.testRequest.version, 'HTTP/1.0')
  754.         self.assertEquals(self.testRequest.status, 200)
  755.         self.assertEquals(self.testRequest.reason, 'OK')
  756.         self.assertEquals(self.testRequest.headers['server'], 'Apache')
  757.         self.assertEquals(self.testRequest.body, 'HELLO: WORLD\r\n')
  758.  
  759.     
  760.     def testOneChunk(self):
  761.         self.testRequest.handleData(self.fakeResponse)
  762.         self.testRequest.handleClose(socket.SHUT_RD)
  763.         self.assertEquals(self.testRequest.version, 'HTTP/1.0')
  764.         self.assertEquals(self.testRequest.status, 200)
  765.         self.assertEquals(self.testRequest.reason, 'OK')
  766.         self.assertEquals(self.testRequest.headers['server'], 'Apache')
  767.         self.assertEquals(self.testRequest.body, 'HELLO: WORLD\r\n')
  768.  
  769.     
  770.     def testBadChunkSize(self):
  771.         self.testRequest.handleData(startResponse(headers = {
  772.             'Transfer-Encoding': 'chunked' }))
  773.         self.testRequest.handleData('Fifty\r\n')
  774.         self.assert_(self.errbackCalled)
  775.         self.assert_(isinstance(self.data, httpclient.BadChunkSize))
  776.  
  777.     
  778.     def testIgnoreChunkExtensions(self):
  779.         self.testRequest.handleData(startResponse(headers = {
  780.             'Transfer-Encoding': 'chunked' }))
  781.         self.testRequest.handleData('ff ;ext1=2 ; ext3=4\r\n')
  782.         self.assert_(not (self.errbackCalled))
  783.         self.assertEquals(self.testRequest.state, 'chunk-data')
  784.         self.assertEquals(self.testRequest.chunkSize, 255)
  785.  
  786.     
  787.     def testChunkWithoutCRLF(self):
  788.         self.testRequest.handleData(startResponse(headers = {
  789.             'Transfer-Encoding': 'chunked' }))
  790.         self.testRequest.handleData('5\r\n')
  791.         self.testRequest.handleData('12345RN')
  792.         self.assert_(self.errback)
  793.         self.assert_(isinstance(self.data, httpclient.CRLFExpected))
  794.  
  795.     
  796.     def testPrematureClose(self):
  797.         data = self.fakeResponse
  798.         self.testRequest.handleData(data[:123])
  799.         self.testRequest.handleClose(socket.SHUT_RD)
  800.         self.assert_(self.errbackCalled)
  801.         self.assert_(isinstance(self.data, httpclient.ServerClosedConnection))
  802.  
  803.  
  804.  
  805. class HTTPClientTestBase(AsyncSocketTest):
  806.     
  807.     def setUp(self):
  808.         AsyncSocketTest.setUp(self)
  809.         self.testRequest = TestingHTTPConnection()
  810.         self.testRequest.openConnection('foo.com', 80, (lambda x: None), (lambda x: None))
  811.         self.testRequest.sendRequest(self.callback, self.errback, '', 80, method = 'GET', path = '/bar/baz;123?a=b')
  812.         self.authDelegate = TestingAuthDelegate()
  813.         dialogs.setDelegate(self.authDelegate)
  814.         TestHTTPClient.connectionPool = TestingHTTPConnectionPool()
  815.         TestingHeaderGrabber.connectionPool = TestingHTTPConnectionPool()
  816.  
  817.     
  818.     def tearDown(self):
  819.         AsyncSocketTest.tearDown(self)
  820.  
  821.  
  822.  
  823. class HTTPClientTest(HTTPClientTestBase):
  824.     
  825.     def testRealRequest(self):
  826.         url = 'http://participatoryculture.org/democracytest/normalpage.txt'
  827.         httpclient.grabURL(url, self.callback, self.errback, clientClass = TestHTTPClient)
  828.         self.runEventLoop(timeout = 10)
  829.         self.assert_(self.callbackCalled)
  830.         self.assertEquals(self.data['body'], 'I AM A NORMAL PAGE\n')
  831.  
  832.     
  833.     def testGrabHeaders(self):
  834.         url = 'http://participatoryculture.org/democracytest/normalpage.txt'
  835.         httpclient.grabHeaders(url, self.callback, self.errback, TestingHeaderGrabber)
  836.         self.runEventLoop(timeout = 10)
  837.         self.assert_(self.callbackCalled)
  838.         self.assertEquals(self.data['body'], '')
  839.         self.assertEquals(self.data['status'], 200)
  840.         self.assertEquals(self.data['original-url'], self.data['updated-url'])
  841.         self.assertEquals(self.data['original-url'], self.data['redirected-url'])
  842.  
  843.     
  844.     def testGrabHeaders2(self):
  845.         url = 'http://participatoryculture.org/democracytest/nohead.php'
  846.         httpclient.grabHeaders(url, self.callback, self.errback, clientClass = TestingHeaderGrabber)
  847.         self.runEventLoop(timeout = 10)
  848.         self.assert_(self.callbackCalled)
  849.         self.assertEquals(self.data['body'], '')
  850.         self.assertEquals(self.data['status'], 200)
  851.         self.assertEquals(self.data['original-url'], self.data['updated-url'])
  852.         self.assertEquals(self.data['original-url'], self.data['redirected-url'])
  853.  
  854.     
  855.     def testGrabHeadersCancel(self):
  856.         url = 'http://participatoryculture.org/democracytest/normalpage.txt'
  857.         client = httpclient.grabHeaders(url, self.callback, self.errback, clientClass = TestingHeaderGrabber)
  858.         client.cancel()
  859.         self.assertRaises((HadToStopEventLoop,), (lambda : self.runEventLoop(timeout = 1)))
  860.  
  861.     
  862.     def testConnectionFailure(self):
  863.         httpclient.grabURL('http://slashdot.org:123123', self.callback, self.errback, clientClass = TestHTTPClient)
  864.         self.runEventLoop()
  865.         self.assert_(self.errbackCalled)
  866.         self.assertEquals(self.data.__class__, httpclient.ConnectionError)
  867.  
  868.     
  869.     def testMultipleRequests(self):
  870.         
  871.         def middleCallback(data):
  872.             self.firstData = data
  873.             req.sendRequest(self.callback, self.errback, 'participatoryculture.org', 80, method = 'GET', path = '/democracytest/normalpage.txt')
  874.  
  875.         req = TestingHTTPConnection()
  876.         
  877.         def stopEventLoop(conn):
  878.             self.stopEventLoop(False)
  879.  
  880.         (None, ((None, None),), self.addIdle)((lambda : req.openConnection('participatoryculture.org', 80, stopEventLoop, self.errback)), 'Open connection')
  881.         self.runEventLoop()
  882.         self.assert_(not (self.errbackCalled))
  883.         (None, None, self.addIdle)((lambda : req.sendRequest(middleCallback, self.errback, 'participatoryculture.org', 80, method = 'GET', path = '/democracytest/normalpage.txt')), 'Send Request')
  884.         self.runEventLoop()
  885.         self.assert_(not (self.errbackCalled))
  886.         self.assertEquals(self.firstData['body'], self.data['body'])
  887.  
  888.     
  889.     def testUnexpectedStatusCode(self):
  890.         """Test what happens when we get a bad status code.  
  891.         
  892.         The header callback should be called, but the on body data callback
  893.         shouldn't.  Also, we should call the errback instead of the callback.
  894.         """
  895.         self.onHeadersCalled = self.onBodyDataCalled = False
  896.         
  897.         def onHeaders(headers):
  898.             self.onHeadersCalled = True
  899.  
  900.         
  901.         def onBodyData(data):
  902.             self.onBodyDataCalled = True
  903.  
  904.         url = 'http://participatoryculture.org/404'
  905.         client = TestHTTPClient(url, self.callback, self.errback, onHeaders, onBodyData)
  906.         client.startRequest()
  907.         self.runEventLoop()
  908.         self.assert_(self.onHeadersCalled)
  909.         self.assert_(not (self.onBodyDataCalled))
  910.         self.assert_(not (self.callbackCalled))
  911.         self.assert_(self.errbackCalled)
  912.  
  913.     
  914.     def testHeaderCallback(self):
  915.         
  916.         def headerCallback(response):
  917.             self.headerResponse = copy(response)
  918.             self.callbackCalledInHeaderCallback = self.callbackCalled
  919.             self.errbackCalledInHeaderCallback = self.errbackCalled
  920.             self.stopEventLoop(False)
  921.  
  922.         url = 'http://participatoryculture.org/democracytest/normalpage.txt'
  923.         httpclient.grabURL(url, self.callback, self.errback, headerCallback = headerCallback, clientClass = TestHTTPClient)
  924.         self.runEventLoop()
  925.         self.assert_(not (self.callbackCalledInHeaderCallback))
  926.         self.assert_(not (self.errbackCalledInHeaderCallback))
  927.         self.assert_(self.headerResponse['content-type'].startswith('text/plain'))
  928.         self.assertEquals(self.headerResponse['body'], None)
  929.  
  930.     
  931.     def testHeaderCallbackCancel(self):
  932.         
  933.         def headerCallback(response):
  934.             reqId.cancel()
  935.             self.stopEventLoop(False)
  936.  
  937.         url = 'http://participatoryculture.org/democracytest/normalpage.txt'
  938.         reqId = httpclient.grabURL(url, self.callback, self.errback, headerCallback = headerCallback, clientClass = TestHTTPClient)
  939.         self.failedCalled = False
  940.         
  941.         def fakeFailed(*args, **kwargs):
  942.             self.failedCalled = True
  943.  
  944.         oldFailed = util.failed
  945.         util.failed = fakeFailed
  946.         self.runEventLoop()
  947.         util.failed = oldFailed
  948.         self.assert_(not (self.callbackCalled))
  949.         self.assert_(not (self.errbackCalled))
  950.         self.assert_(not (self.failedCalled))
  951.  
  952.     
  953.     def testBodyDataCallbackCancel(self):
  954.         
  955.         def bodyDataCallback(response):
  956.             reqId.cancel()
  957.             self.stopEventLoop(False)
  958.  
  959.         url = 'http://participatoryculture.org/democracytest/normalpage.txt'
  960.         reqId = httpclient.grabURL(url, self.callback, self.errback, bodyDataCallback = bodyDataCallback, clientClass = TestHTTPClient)
  961.         self.failedCalled = False
  962.         
  963.         def fakeFailed(*args, **kwargs):
  964.             self.failedCalled = True
  965.  
  966.         oldFailed = util.failed
  967.         util.failed = fakeFailed
  968.         self.runEventLoop()
  969.         util.failed = oldFailed
  970.         self.assert_(not (self.callbackCalled))
  971.         self.assert_(not (self.errbackCalled))
  972.         self.assert_(not (self.failedCalled))
  973.  
  974.     
  975.     def testBodyDataCallback(self):
  976.         self.lastSeen = None
  977.         
  978.         def bodyDataCallback(data):
  979.             self.lastSeen = data
  980.  
  981.         self.testRequest.bodyDataCallback = bodyDataCallback
  982.         self.testRequest.handleData(startResponse(headers = {
  983.             'content-length': '20' }))
  984.         self.assertEquals(self.lastSeen, None)
  985.         self.testRequest.handleData('12345')
  986.         self.assertEquals(self.lastSeen, '12345')
  987.         self.testRequest.handleData('1234567890')
  988.         self.assertEquals(self.lastSeen, '1234567890')
  989.         self.testRequest.handleData('1234567890')
  990.         self.assertEquals(self.lastSeen, '12345')
  991.  
  992.     
  993.     def testBodyDataCallbackChunked(self):
  994.         self.lastSeen = None
  995.         
  996.         def bodyDataCallback(data):
  997.             self.lastSeen = data
  998.  
  999.         self.testRequest.bodyDataCallback = bodyDataCallback
  1000.         self.testRequest.handleData(startResponse(headers = {
  1001.             'transfer-encoding': 'chunked' }))
  1002.         self.testRequest.handleData('5\r\nHI')
  1003.         self.assertEquals(self.lastSeen, 'HI')
  1004.         self.testRequest.handleData('BEN\r\n')
  1005.         self.assertEquals(self.lastSeen, 'BEN')
  1006.         self.testRequest.handleData('A\r\n')
  1007.         self.assertEquals(self.lastSeen, 'BEN')
  1008.         self.testRequest.handleData('1234567890\r\n0')
  1009.         self.assertEquals(self.lastSeen, '1234567890')
  1010.         self.assert_(not (self.callbackCalled))
  1011.         self.testRequest.handleData('\r\n\r\n')
  1012.         self.assert_(self.callbackCalled)
  1013.  
  1014.     
  1015.     def testBodyDataCallbackRealRequest(self):
  1016.         url = 'http://participatoryculture.org/democracytest/normalpage.txt'
  1017.         self.gotData = ''
  1018.         
  1019.         def bodyDataCallback(data):
  1020.             self.gotData += data
  1021.             if self.gotData == 'I AM A NORMAL PAGE\n':
  1022.                 self.stopEventLoop(False)
  1023.             
  1024.  
  1025.         httpclient.grabURL(url, self.callback, self.errback, bodyDataCallback = bodyDataCallback, clientClass = TestHTTPClient)
  1026.         self.runEventLoop()
  1027.         self.assertEquals(self.gotData, 'I AM A NORMAL PAGE\n')
  1028.  
  1029.     
  1030.     def testAuth(self):
  1031.         self.authDelegate.addLogin(u'ben', u'baddpassword')
  1032.         self.authDelegate.addLogin(u'guest', u'guest')
  1033.         url = 'http://jigsaw.w3.org/HTTP/Basic/'
  1034.         client = TestHTTPClient(url, self.callback, self.errback)
  1035.         client.startRequest()
  1036.         self.runEventLoop()
  1037.         self.assert_(self.callbackCalled)
  1038.         self.assertEquals(self.data['status'], 200)
  1039.         self.assertEquals(client.authAttempts, 2)
  1040.  
  1041.     
  1042.     def testBadAuth(self):
  1043.         self.authDelegate.addLogin(u'baduser', u'baddpass')
  1044.         self.authDelegate.addLogin(u'anotherbadtry', u'god')
  1045.         self.authDelegate.addLogin(u'billgates', u'password')
  1046.         url = 'http://jigsaw.w3.org/HTTP/Basic/'
  1047.         client = TestHTTPClient(url, self.callback, self.errback)
  1048.         client.startRequest()
  1049.         self.runEventLoop()
  1050.         self.assert_(self.errbackCalled)
  1051.         self.assert_(isinstance(self.data, httpclient.AuthorizationFailed))
  1052.         self.assertEquals(client.authAttempts, 3)
  1053.  
  1054.     
  1055.     def testDigestAuth(self):
  1056.         url = 'http://jigsaw.w3.org/HTTP/Digest/'
  1057.         client = TestHTTPClient(url, self.callback, self.errback)
  1058.         client.startRequest()
  1059.         self.runEventLoop()
  1060.         self.assert_(self.errbackCalled)
  1061.         self.assert_(isinstance(self.data, httpclient.AuthorizationFailed))
  1062.         self.assertEquals(client.authAttempts, 0)
  1063.  
  1064.     
  1065.     def testParseURL(self):
  1066.         (scheme, host, port, path) = httpclient.parseURL('https://www.foo.com/abc;123?a=b#4')
  1067.         self.assertEquals(scheme, 'https')
  1068.         self.assertEquals(host, 'www.foo.com')
  1069.         self.assertEquals(port, 443)
  1070.         self.assertEquals(path, '/abc;123?a=b')
  1071.         (scheme, host, port, path) = httpclient.parseURL('http://www.foo.com/abc;123?a=b#4')
  1072.         self.assertEquals(port, 80)
  1073.         (scheme, host, port, path) = httpclient.parseURL('http://www.foo.com:5000/abc;123?a=b#4')
  1074.         self.assertEquals(port, 5000)
  1075.         (scheme, host, port, path) = httpclient.parseURL('http://www.foo.com:123:123/abc;123?a=b#4')
  1076.         self.assertEquals(port, 123)
  1077.  
  1078.     
  1079.     def testCleanFilename(self):
  1080.         tempdir = tempfile.gettempdir()
  1081.         
  1082.         def testIt(filename):
  1083.             cleaned = cleanFilename(filename)
  1084.             self.assertEqual(cleaned.__class__, str)
  1085.             self.assertNotEqual(cleaned, '')
  1086.             path = os.path.join(tempdir, cleaned)
  1087.             f = open(path, 'w')
  1088.             f.write('AOEUOAEU')
  1089.             f.close()
  1090.             os.remove(path)
  1091.  
  1092.         testIt(u'iamnormal.txt')
  1093.         testIt(u'???')
  1094.         testIt(u'├╕benben.jpg')
  1095.         testIt(u'├╕???.├▓├│x')
  1096.  
  1097.     
  1098.     def testGetFilenameFromResponse(self):
  1099.         client = TestHTTPClient('http://www.foo.com', self.callback, self.errback)
  1100.         
  1101.         def getIt(path, cd = (None,)):
  1102.             response = {
  1103.                 'path': path }
  1104.             if cd:
  1105.                 response['content-disposition'] = cd
  1106.             
  1107.             return client.getFilenameFromResponse(response)
  1108.  
  1109.         self.assertEquals('unknown', getIt('/'))
  1110.         self.assertEquals('index.html', getIt('/index.html'))
  1111.         self.assertEquals('index.html', getIt('/path/path2/index.html'))
  1112.         self.assertEquals('unknown', getIt('/path/path2/'))
  1113.         self.assertEquals('myfile.txt', getIt('/', 'filename="myfile.txt"'))
  1114.         self.assertEquals('myfile.txt', getIt('/', 'filename="myfile.txt"; size=45'))
  1115.         self.assertEquals('myfile.txt', getIt('/', ' filename =  "myfile.txt"'))
  1116.         self.assertEquals('myfile.txt', getIt('/', 'filename=myfile.txt'))
  1117.         self.assertEquals('myfile.txt', getIt('/index.html', 'filename="myfile.txt"'))
  1118.         self.assertEquals('lots.of.extensions', getIt('/', 'filename="lots.of.extensions"'))
  1119.         self.assertEquals('uncleanfilename', getIt('/index', 'filename="\\un/cl:ean*fi?lena<m>|e"'))
  1120.         self.assertEquals('uncleanfil-ename2', getIt('/uncl*ean"fil?"ena|m""e2"'))
  1121.  
  1122.     
  1123.     def testGetCharsetFromResponse(self):
  1124.         client = TestHTTPClient('http://participatoryculture.org/democracytest/normal.txt', self.callback, self.errback)
  1125.         
  1126.         def getIt(contentType):
  1127.             if contentType:
  1128.                 response = {
  1129.                     'content-type': contentType }
  1130.             else:
  1131.                 response = { }
  1132.             return client.getCharsetFromResponse(response)
  1133.  
  1134.         self.assertEquals('iso-8859-1', getIt(None))
  1135.         self.assertEquals('iso-8859-1', getIt('gabaldigook'))
  1136.         self.assertEquals('iso-8859-1', getIt('text/html'))
  1137.         self.assertEquals('utf-8', getIt('text/html; charset=utf-8'))
  1138.         self.assertEquals('utf-8', getIt('text/html; charset = utf-8'))
  1139.         self.assertEquals('utf-8', getIt('text/html; charset=utf-8; extraparam=2'))
  1140.  
  1141.  
  1142.  
  1143. class HTTPConnectionPoolTest(EventLoopTest):
  1144.     
  1145.     def setUp(self):
  1146.         self.pool = TestingHTTPConnectionPool()
  1147.         super(HTTPConnectionPoolTest, self).setUp()
  1148.  
  1149.     
  1150.     def addRequest(self, url):
  1151.         return (self.pool.addRequest,)((lambda blah: (self.addIdle,)((lambda : self.stopEventLoop(False)), 'Closing connection when request is done')
  1152. ), (lambda error: 0), None, None, None, url, 'GET', { })
  1153.  
  1154.     
  1155.     def checkCounts(self, activeCount, freeCount, pendingCount):
  1156.         self.assertEquals(self.pool.activeConnectionCount, activeCount)
  1157.         self.assertEquals(self.pool.freeConnectionCount, freeCount)
  1158.         realFreeCount = realActiveCount = 0
  1159.         for key, conns in self.pool.connections.items():
  1160.             realFreeCount += len(conns['free'])
  1161.             realActiveCount += len(conns['active'])
  1162.         
  1163.         self.assertEquals(realActiveCount, activeCount)
  1164.         self.assertEquals(realFreeCount, freeCount)
  1165.         self.assertEquals(pendingCount, len(self.pool.pendingRequests))
  1166.  
  1167.     
  1168.     def testNormalUsage(self):
  1169.         self.addRequest('http://www.foo.com/')
  1170.         self.addRequest('http://www.bar.com/')
  1171.         self.addRequest('http://www.foo.com/2')
  1172.         self.addRequest('http://www.google.com/')
  1173.         self.checkCounts(4, 0, 0)
  1174.  
  1175.     
  1176.     def testOpenConnectionFailed(self):
  1177.         self.pool = TestingHTTPConnectionPool()
  1178.         
  1179.         def stopEventLoop(error):
  1180.             self.stopEventLoop(False)
  1181.  
  1182.         self.pool.addRequest(stopEventLoop, stopEventLoop, None, None, None, 'http://uselessurl/', 'GET', { })
  1183.         self.runEventLoop()
  1184.         self.checkCounts(0, 0, 0)
  1185.  
  1186.     
  1187.     def testCounts(self):
  1188.         self.addRequest('http://participatoryculture.org/')
  1189.         self.addRequest('http://participatoryculture.org/democracytest/normalpage.txt')
  1190.         self.addRequest('https://participatoryculture.org/')
  1191.         self.checkCounts(3, 0, 0)
  1192.         self.runEventLoop()
  1193.         self.checkCounts(2, 1, 0)
  1194.         self.pool.closeConnection('https', 'participatoryculture.org')
  1195.         self.checkCounts(1, 1, 0)
  1196.         self.pool.closeConnection('http', 'participatoryculture.org', type = 'free')
  1197.         self.checkCounts(1, 0, 0)
  1198.  
  1199.     
  1200.     def testServerLimit(self):
  1201.         self.addRequest('http://participatoryculture.org/democracytest/normalpage.txt')
  1202.         self.addRequest('http://participatoryculture.org/democracytest/normalpage2.txt')
  1203.         self.addRequest('https://participatoryculture.org/democracytest/normalpage.txt')
  1204.         self.checkCounts(3, 0, 0)
  1205.         self.addRequest('http://participatoryculture.org/democracytest/normalpage3.txt')
  1206.         self.checkCounts(3, 0, 1)
  1207.         self.pool.assertConnectionNotStarted('http://participatoryculture.org/democracytest/normalpage3.txt')
  1208.         self.runEventLoop()
  1209.         self.checkCounts(3, 0, 0)
  1210.         self.runEventLoop()
  1211.         self.pool.assertConnectionStarted('http://participatoryculture.org/democracytest/normalpage3.txt')
  1212.         self.checkCounts(2, 1, 0)
  1213.  
  1214.     
  1215.     def testTotalLimit(self):
  1216.         self.addRequest('http://participatoryculture.org/democracytest/normalpage.txt')
  1217.         self.addRequest('http://participatoryculture.org/democracytest/normalpage2.txt')
  1218.         self.addRequest('http://www.bar.com/')
  1219.         self.addRequest('http://www.bar.com/2')
  1220.         self.addRequest('http://www.baz.com/')
  1221.         self.addRequest('http://www.froz.com/')
  1222.         self.checkCounts(4, 0, 2)
  1223.  
  1224.     
  1225.     def testBothLimits(self):
  1226.         self.addRequest('http://participatoryculture.org/')
  1227.         self.addRequest('http://participatoryculture.org/2')
  1228.         self.addRequest('http://participatoryculture.org/3')
  1229.         self.checkCounts(2, 0, 1)
  1230.         self.pool.assertConnectionNotStarted('http://participatoryculture.org/3')
  1231.         self.addRequest('https://www.bar.com/')
  1232.         self.addRequest('http://www.bar.com/2')
  1233.         self.addRequest('http://www.baz.com/')
  1234.         self.checkCounts(4, 0, 2)
  1235.         self.pool.assertConnectionNotStarted('http://www.baz.com/')
  1236.         self.runEventLoop()
  1237.         self.checkCounts(4, 0, 1)
  1238.         self.pool.assertConnectionNotStarted('http://www.baz.com/')
  1239.         self.runEventLoop()
  1240.         self.checkCounts(4, 0, 0)
  1241.         self.runEventLoop()
  1242.  
  1243.     
  1244.     def testDropTheLRU(self):
  1245.         self.addRequest('http://www.baz.com/')
  1246.         self.addRequest('http://participatoryculture.org/')
  1247.         self.addRequest('http://www.bar.com/')
  1248.         self.addRequest('http://www.froz.com/')
  1249.         self.addRequest('http://www.qux.com/')
  1250.         if len(self.pool.connections['http:www.baz.com:80']['free']) == 0:
  1251.             pass
  1252.         self.assert_(len(self.pool.connections['http:www.baz.com:80']['active']) > 0)
  1253.         self.runEventLoop()
  1254.         if len(self.pool.connections['http:www.baz.com:80']['free']) == 0:
  1255.             pass
  1256.         self.assert_(len(self.pool.connections['http:www.baz.com:80']['active']) == 0)
  1257.  
  1258.  
  1259.  
  1260. class GrabURLTest(AsyncSocketTest):
  1261.     
  1262.     def testStart(self):
  1263.         url = 'http://participatoryculture.org/democracytest/normalpage.txt'
  1264.         httpclient.grabURL(url, self.callback, self.errback, clientClass = TestHTTPClient)
  1265.         self.runEventLoop()
  1266.         self.origData = self.data
  1267.         httpclient.grabURL(url, self.callback, self.errback, start = 4, clientClass = TestHTTPClient)
  1268.         self.runEventLoop()
  1269.         self.assertEquals(self.data['body'], self.origData['body'])
  1270.         self.assertEquals(self.data['status'], 200)
  1271.  
  1272.  
  1273.  
  1274. class BadURLTest(HTTPClientTestBase):
  1275.     
  1276.     def testScheme(self):
  1277.         url = 'participatoryculture.org/democracytest/normalpage.txt'
  1278.         httpclient.grabURL(url, self.callback, self.errback, clientClass = TestHTTPClient)
  1279.         self.runPendingIdles()
  1280.         self.assertEquals(self.errbackCalled, True)
  1281.         self.assertEquals(self.callbackCalled, False)
  1282.  
  1283.     
  1284.     def testSlashes(self):
  1285.         url = 'http:jigsaw.w3.org/HTTP/'
  1286.         httpclient.grabURL(url, self.callback, self.errback, clientClass = TestHTTPClient)
  1287.         self.runPendingIdles()
  1288.         self.assertEquals(self.errbackCalled, True)
  1289.         self.assertEquals(self.callbackCalled, False)
  1290.  
  1291.     
  1292.     def testHost(self):
  1293.         url = 'http:///HTTP/'
  1294.         httpclient.grabURL(url, self.callback, self.errback, clientClass = TestHTTPClient)
  1295.         self.runPendingIdles()
  1296.         self.assertEquals(self.errbackCalled, True)
  1297.         self.assertEquals(self.callbackCalled, False)
  1298.  
  1299.     
  1300.     def testOtherScheme(self):
  1301.         url = 'rtsp://jigsaw.w3.org/'
  1302.         httpclient.grabURL(url, self.callback, self.errback, clientClass = TestHTTPClient)
  1303.         self.runPendingIdles()
  1304.         self.assertEquals(self.errbackCalled, True)
  1305.         self.assertEquals(self.callbackCalled, False)
  1306.  
  1307.  
  1308.  
  1309. class CookieExpirationDateTestCase(unittest.TestCase):
  1310.     
  1311.     def testCookieExpirationDate(self):
  1312.         """Tests get_cookie_expiration_date to make sure it's returning
  1313.         sane values and handles cookie expiration formats we've had
  1314.         problems with.
  1315.         """
  1316.         mktime = mktime
  1317.         strptime = strptime
  1318.         localtime = localtime
  1319.         import time
  1320.         get_cookie_expiration_date = get_cookie_expiration_date
  1321.         import httpclient
  1322.         for cd in (('Thu, 03-May-07 22:48:52 GMT', '2007-05-03 22:48:52 GMT'), ('Fri, 03-Jun-11 13:41:15 GMT', '2011-06-03 13:41:15 GMT'), ('Sun, 17-Jan-2038 19:14:07 GMT', '2038-01-17 19:14:07 GMT'), ('Mon, 09-Apr-07 23:50:49 GMT', '2007-04-09 23:50:49 GMT'), ('Tue, 01-Jan-2030 10:00:00 GMT', '2030-01-01 10:00:00 GMT'), ('Tue, 17-Jul-2007 02:09:00 GMT', '2007-07-17 02:09:00 GMT')):
  1323.             self.assertEquals(localtime(get_cookie_expiration_date(cd[0])), localtime(mktime(strptime(cd[1], '%Y-%m-%d %H:%M:%S %Z'))))
  1324.         
  1325.  
  1326.     
  1327.     def testOverflowCookieExpirationDate(self):
  1328.         '''tests the case of get_cookie_expiration_date where the cookie
  1329.         expiration date causes an overflow error when parsing it.
  1330.         '''
  1331.         localtime = localtime
  1332.         import time
  1333.         get_cookie_expiration_date = get_cookie_expiration_date
  1334.         DATEINFUTURE = DATEINFUTURE
  1335.         import httpclient
  1336.         self.assertEquals(localtime(get_cookie_expiration_date('Tue, 26-Jul-2050 10:00:00 GMT')), localtime(httpclient.DATEINFUTURE))
  1337.  
  1338.  
  1339.